package com.share.example.spider.processor;

import com.share.example.spider.model.db.Stock;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Date;
import java.util.LinkedList;
import java.util.List;

/**
 * @author 01401061 - 孙艳强
 * @date 2020/10/28 15:26
 */
@Slf4j
public class StockProcessor implements PageProcessor {


    /**
     * 1.解析数据
     * 2.获取下一页的链接
     * 3.通过page把数据推出去
     */
    @Override
    public void process(Page page) {
        Html html = page.getHtml();

        List<Stock> list = new LinkedList();

        List<Selectable> nodes = html.xpath("/html/body/table/tbody/tr").nodes();
        for (Selectable node : nodes) {
            String code1 = node.xpath("/tr/td[1]/text()").get();
            String name1 = node.xpath("/tr/td[2]/text()").get();
            String code2 = node.xpath("/tr/td[3]/text()").get();
            String name2 = node.xpath("/tr/td[4]/text()").get();

            if (StringUtils.isNotBlank(code1) && StringUtils.isNotBlank(name1)) {
                Stock data1 = new Stock();
                data1.setCode(code1.trim());
                data1.setName(name1.trim());
                data1.setStatus("1");
                data1.setCreateTime(new Date());
                list.add(data1);
            }

            if (StringUtils.isNotBlank(code2) && StringUtils.isNotBlank(name2)) {
                Stock data2 = new Stock();
                data2.setCode(code2.trim());
                data2.setName(name2.trim());
                data2.setStatus("1");
                data2.setCreateTime(new Date());
                list.add(data2);
            }
        }

        page.putField(Stock.class.getName(), list);
    }

    /*site的作用是限定爬取的网络位置*/
    @Override
    public Site getSite() {
        Site site = new Site();
        site.setDomain("www.txsec.com");
        site.setUserAgent("User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36");
        return site;
    }
}
